
# Create a temporary directory to hold notebook output, and change the working directory to that directory.
import shutil
import os
from tempfile import TemporaryDirectory

use_tempdir = True
if use_tempdir:
    # Create temporary directory
    data_dir = TemporaryDirectory()
    # Save original directory
    orig_dir = os.getcwd()
    # Move to new directory
    os.chdir(data_dir.name)

# For info, print out where the script is running
print("Running in {}".format(os.getcwd()))

Running in /internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp5lkmcroh


import os
if 'CRDS_CACHE_TYPE' in os.environ:
    if os.environ['CRDS_CACHE_TYPE'] == 'local':
        os.environ['CRDS_PATH'] = os.path.join(os.environ['HOME'], 'crds', 'cache')
    elif os.path.isdir(os.environ['CRDS_CACHE_TYPE']):
        os.environ['CRDS_PATH'] = os.environ['CRDS_CACHE_TYPE']
print('CRDS cache location: {}'.format(os.environ['CRDS_PATH']))

CRDS cache location: /grp/crds/cache


import warnings
import psutil
from astropy.io import fits

# Only print a DeprecationWarning the first time it shows up, not every time.
with warnings.catch_warnings():
    warnings.simplefilter("once", category=DeprecationWarning)
    import jwst
    from jwst.pipeline.calwebb_detector1 import Detector1Pipeline
    from jwst.pipeline.calwebb_spec2 import Spec2Pipeline
    from jwst.assign_wcs.assign_wcs_step import AssignWcsStep
    from jwst.msaflagopen.msaflagopen_step import MSAFlagOpenStep
    from jwst.extract_2d.extract_2d_step import Extract2dStep
    from jwst.srctype.srctype_step import SourceTypeStep
    from jwst.wavecorr.wavecorr_step import WavecorrStep
    from jwst.flatfield.flat_field_step import FlatFieldStep

# The latest version of NPTT is installed in the requirements text file at:
# /jwst_validation_notebooks/environment.yml

# import NPTT
import nirspec_pipe_testing_tool as nptt

# To get data from Artifactory
from ci_watson.artifactory_helpers import get_bigdata


# Print versions used for the pipeline and NPTT

pipeline_version = jwst.__version__
nptt_version = nptt.__version__

print("Using jwst pipeline version: ", pipeline_version)
print("Using NPTT version: ", nptt_version)

Using jwst pipeline version:  1.8.2
Using NPTT version:  2.0.1


testing_data = {
                'ifu_prism_clear':{
                                  'uncal_file_nrs1': 'ifu_prism_nrs1_uncal.fits',
                                  'uncal_file_nrs2': 'ifu_prism_nrs2_uncal.fits',
                                  'sflat_nrs1': 'nirspec_IFU_sflat_PRISM_OPAQUE_FLAT5_nrs1_f_01.01.fits',
                                  'sflat_nrs2': 'nirspec_IFU_sflat_PRISM_OPAQUE_FLAT5_nrs2_f_01.01.fits',
                                  'fflat': 'nirspec_IFU_fflat_CLEAR_01.01.fits',
                                  'msa_shutter_config': None }
                }


# define function to pull data from Artifactory
def get_artifactory_file(data_set_dict, detector):
    """This function creates a list with all the files needed per detector to run the test.
    Args:
        data_set_dict: dictionary, contains inputs for a specific mode and configuration
        detector: string, either nrs1 or nrs2
    Returns:
        data: list, contains all files needed to run test
    """
    files2obtain = ['uncal_file_nrs1', 'sflat_nrs1', 'fflat', 'msa_shutter_config']
    data = []
    for file in files2obtain:
        data_file = None
        try: 
            if '_nrs' in file and '2' in detector:
                file = file.replace('_nrs1', '_nrs2')

            data_file = get_bigdata('jwst_validation_notebooks',
                                         'validation_data',
                                         'nirspec_data', 
                                         data_set_dict[file])
        except TypeError:
            data.append(None)
            continue

        data.append(data_file)

    return data


# set the D-flat path (used for all test data)
print('Getting D-Flats from Artifactory...')

dflat_nrs1 = get_bigdata('jwst_validation_notebooks',
                         'validation_data',
                         'nirspec_data', 
                         'nirspec_dflat_nrs1_f_01.03.fits')

dflat_nrs2 = get_bigdata('jwst_validation_notebooks',
                         'validation_data',
                         'nirspec_data', 
                         'nirspec_dflat_nrs2_f_01.03.fits')
print('Got D-flats')

# set NPTT switches for this test and other variables
writefile = False
save_figs = False
show_figs = True
results_dict = {}
detectors = ['nrs1', 'nrs2']

Getting D-Flats from Artifactory...
Got D-flats


# Get data for IFU

for mode_config, data_set_dict in testing_data.items():
    if 'ifu' not in mode_config:
        continue
        
    print('Starting to run pipeline and test for mode: ', mode_config)
    
    for detector in detectors:
        print('Testing files for detector: ', detector)
        data = get_artifactory_file(data_set_dict, detector)
        uncal_file, sflat, fflat, msa_shutter_config = data
        print('Working with uncal_file: ', uncal_file)
        uncal_basename = os.path.basename(uncal_file)
        dflat = dflat_nrs1
        if '2' in detector:
            dflat = dflat_nrs2
        
        # Make sure these keywords are properly set
        filt = fits.getval(uncal_file, 'FILTER')
        if 'OPAQUE' in filt or 'allslits' in uncal_basename.lower():
            if 'clear' in uncal_basename.lower():
                filt = 'CLEAR'
            else:
                l = uncal_basename.split("_")
                for li in l:
                    if 'lp' in li.lower():
                        filt = li
                        break
            fits.setval(uncal_file, 'FILTER', value=filt)
        print('Filter = ', filt)

        # Run the stage 1 pipeline 
        print('Running the detector1 pipeline...')
        rate_object = Detector1Pipeline.call(uncal_file)
        
        # Make sure the FXD_SLIT keyword is set correctly
        try:
            if 'full' in rate_object.meta.instrument.fixed_slit:
                rate_object.meta.instrument.fixed_slit = 'NONE'
        except TypeError:
            print('FXD_SLIT keyword = ', rate_object.meta.instrument.fixed_slit)
        
        # Run the stage 2 pipeline steps
        print('\nRunning the spec2 pipeline...')
        skip_file = False
        try:
            parameter_dict = {"flat_field": {"save_interpolated_flat": True},
                              "pathloss": {"skip": True},
                              "barshadow": {"skip": True},
                              "photom": {"skip": True},
                              "resample_spec": {"skip": True},
                              "cube_build": {"skip": True},
                              "extract_1d": {"skip": True}
                             }
            flat_field_object = Spec2Pipeline.call(rate_object, steps=parameter_dict)
        except:
            #print("No open slits fall on detector ", det)  # usually why assign_wcs crashes
            print("* Spec2 pipeline CRASHED or exited with no output for detector ", detector)
            print("  Skipping test for this file. \n")
            skip_file = True
            
        if not skip_file:   
            # accepted threshold difference with respect to benchmark files
            threshold_diff = 9.999e-5
            if 'prism' in uncal_basename.lower():
                threshold_diff = 9.999e-3
            
            # Run the validation test
            %matplotlib inline
            interpolated_flat = os.path.basename(uncal_file).replace('uncal', 'interpolatedflat')
            print('Running flat field test for IFU...')
            result, result_msg, log_msgs = nptt.calwebb_spec2_pytests.auxiliary_code.flattest_ifu.flattest(
                                                                            flat_field_object,
                                                                            dflat_path=dflat,
                                                                            sflat_path=sflat, 
                                                                            fflat_path=fflat,
                                                                            writefile=writefile,
                                                                            mk_all_slices_plt=False, 
                                                                            show_figs=show_figs, 
                                                                            save_figs=save_figs,
                                                                            interpolated_flat=interpolated_flat,
                                                                            threshold_diff=threshold_diff,
                                                                            debug=False)
        else:
            result, result_msg = 'skipped', 'skipped'

        # Did the test passed 
        print("Did flat_field for ", mode_config, " validation test passed? ", result_msg, "\n\n")
        rd = {uncal_basename: result}
        results_dict.update(rd)
        
        # close all open files
        psutil.Process().open_files()
        closing_files = []
        for fd in psutil.Process().open_files():
            if data_dir.name in fd.path:
                closing_files.append(fd)
        for fd in closing_files:
            try:
                print('Closing file: ', fd, '\n\n')
                open(fd.fd).close()
            except:
                print('File already closed: ', fd, '\n\n')

Starting to run pipeline and test for mode:  ifu_prism_clear
Testing files for detector:  nrs1
Working with uncal_file:  /internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp5lkmcroh/ifu_prism_nrs1_uncal.fits
Filter =  CLEAR
Running the detector1 pipeline...

2022-12-03 20:11:38,823 - stpipe - INFO - PARS-DETECTOR1PIPELINE parameters found: /grp/crds/cache/references/jwst/jwst_nirspec_pars-detector1pipeline_0004.asdf
2022-12-03 20:11:38,857 - stpipe.Detector1Pipeline - INFO - Detector1Pipeline instance created.
2022-12-03 20:11:38,859 - stpipe.Detector1Pipeline.group_scale - INFO - GroupScaleStep instance created.
2022-12-03 20:11:38,861 - stpipe.Detector1Pipeline.dq_init - INFO - DQInitStep instance created.
2022-12-03 20:11:38,862 - stpipe.Detector1Pipeline.saturation - INFO - SaturationStep instance created.
2022-12-03 20:11:38,863 - stpipe.Detector1Pipeline.ipc - INFO - IPCStep instance created.
2022-12-03 20:11:38,865 - stpipe.Detector1Pipeline.superbias - INFO - SuperBiasStep instance created.
2022-12-03 20:11:38,866 - stpipe.Detector1Pipeline.refpix - INFO - RefPixStep instance created.
2022-12-03 20:11:38,867 - stpipe.Detector1Pipeline.rscd - INFO - RscdStep instance created.
2022-12-03 20:11:38,868 - stpipe.Detector1Pipeline.firstframe - INFO - FirstFrameStep instance created.
2022-12-03 20:11:38,869 - stpipe.Detector1Pipeline.lastframe - INFO - LastFrameStep instance created.
2022-12-03 20:11:38,870 - stpipe.Detector1Pipeline.linearity - INFO - LinearityStep instance created.
2022-12-03 20:11:38,873 - stpipe.Detector1Pipeline.dark_current - INFO - DarkCurrentStep instance created.
2022-12-03 20:11:38,874 - stpipe.Detector1Pipeline.reset - INFO - ResetStep instance created.
2022-12-03 20:11:38,876 - stpipe.Detector1Pipeline.persistence - INFO - PersistenceStep instance created.
2022-12-03 20:11:38,877 - stpipe.Detector1Pipeline.jump - INFO - JumpStep instance created.
2022-12-03 20:11:38,878 - stpipe.Detector1Pipeline.ramp_fit - INFO - RampFitStep instance created.
2022-12-03 20:11:38,880 - stpipe.Detector1Pipeline.gain_scale - INFO - GainScaleStep instance created.
2022-12-03 20:11:39,149 - stpipe.Detector1Pipeline - INFO - Step Detector1Pipeline running with args ('/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp5lkmcroh/ifu_prism_nrs1_uncal.fits',).
2022-12-03 20:11:39,159 - stpipe.Detector1Pipeline - INFO - Step Detector1Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_calibrated_ramp': False, 'steps': {'group_scale': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'dq_init': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'saturation': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'n_pix_grow_sat': 1}, 'ipc': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'superbias': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'refpix': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'odd_even_columns': True, 'use_side_ref_pixels': True, 'side_smoothing_length': 11, 'side_gain': 1.0, 'odd_even_rows': True}, 'rscd': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'type': 'baseline'}, 'firstframe': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'lastframe': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'linearity': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'dark_current': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'dark_output': None}, 'reset': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'persistence': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'input_trapsfilled': '', 'flag_pers_cutoff': 40.0, 'save_persistence': False, 'save_trapsfilled': True}, 'jump': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'rejection_threshold': 4.0, 'three_group_rejection_threshold': 6.0, 'four_group_rejection_threshold': 5.0, 'maximum_cores': 'none', 'flag_4_neighbors': True, 'max_jump_to_flag_neighbors': 1000.0, 'min_jump_to_flag_neighbors': 10.0, 'after_jump_flag_dn1': 0.0, 'after_jump_flag_time1': 0.0, 'after_jump_flag_dn2': 0.0, 'after_jump_flag_time2': 0.0, 'min_sat_area': 1.0, 'min_jump_area': 5.0, 'expand_factor': 2.0, 'use_ellipses': False, 'sat_required_snowball': True, 'expand_large_events': False}, 'ramp_fit': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'int_name': '', 'save_opt': False, 'opt_name': '', 'suppress_one_group': True, 'maximum_cores': 'none'}, 'gain_scale': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}}}
2022-12-03 20:11:39,400 - stpipe.Detector1Pipeline - INFO - Prefetching reference files for dataset: 'ifu_prism_nrs1_uncal.fits' reftypes = ['dark', 'gain', 'linearity', 'mask', 'persat', 'readnoise', 'refpix', 'reset', 'rscd', 'saturation', 'superbias', 'trapdensity', 'trappars']
2022-12-03 20:11:39,538 - stpipe.Detector1Pipeline - INFO - Prefetch for DARK reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_dark_0140.fits'.
2022-12-03 20:11:39,551 - stpipe.Detector1Pipeline - INFO - Prefetch for GAIN reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_gain_0019.fits'.
2022-12-03 20:11:39,555 - stpipe.Detector1Pipeline - INFO - Prefetch for LINEARITY reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_linearity_0018.fits'.
2022-12-03 20:11:39,560 - stpipe.Detector1Pipeline - INFO - Prefetch for MASK reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_mask_0024.fits'.
2022-12-03 20:11:39,571 - stpipe.Detector1Pipeline - INFO - Prefetch for PERSAT reference file is 'N/A'.
2022-12-03 20:11:39,571 - stpipe.Detector1Pipeline - INFO - Prefetch for READNOISE reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_readnoise_0019.fits'.
2022-12-03 20:11:39,579 - stpipe.Detector1Pipeline - INFO - Prefetch for REFPIX reference file is 'N/A'.
2022-12-03 20:11:39,579 - stpipe.Detector1Pipeline - INFO - Prefetch for RESET reference file is 'N/A'.
2022-12-03 20:11:39,580 - stpipe.Detector1Pipeline - INFO - Prefetch for RSCD reference file is 'N/A'.
2022-12-03 20:11:39,580 - stpipe.Detector1Pipeline - INFO - Prefetch for SATURATION reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_saturation_0020.fits'.
2022-12-03 20:11:39,587 - stpipe.Detector1Pipeline - INFO - Prefetch for SUPERBIAS reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_superbias_0087.fits'.
2022-12-03 20:11:39,603 - stpipe.Detector1Pipeline - INFO - Prefetch for TRAPDENSITY reference file is 'N/A'.
2022-12-03 20:11:39,604 - stpipe.Detector1Pipeline - INFO - Prefetch for TRAPPARS reference file is 'N/A'.
2022-12-03 20:11:39,604 - stpipe.Detector1Pipeline - INFO - Starting calwebb_detector1 ...
2022-12-03 20:11:40,251 - stpipe.Detector1Pipeline.group_scale - INFO - Step group_scale running with args (<RampModel(1, 10, 2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:11:40,253 - stpipe.Detector1Pipeline.group_scale - INFO - Step group_scale parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp5lkmcroh'}
2022-12-03 20:11:40,494 - stpipe.Detector1Pipeline.group_scale - INFO - NFRAMES and FRMDIVSR are equal; correction not needed
2022-12-03 20:11:40,495 - stpipe.Detector1Pipeline.group_scale - INFO - Step will be skipped
2022-12-03 20:11:40,498 - stpipe.Detector1Pipeline.group_scale - INFO - Step group_scale done
2022-12-03 20:11:40,739 - stpipe.Detector1Pipeline.dq_init - INFO - Step dq_init running with args (<RampModel(1, 10, 2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:11:40,741 - stpipe.Detector1Pipeline.dq_init - INFO - Step dq_init parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp5lkmcroh'}
2022-12-03 20:11:40,768 - stpipe.Detector1Pipeline.dq_init - INFO - Using MASK reference file /grp/crds/cache/references/jwst/jwst_nirspec_mask_0024.fits
2022-12-03 20:11:42,214 - stpipe.Detector1Pipeline.dq_init - INFO - Step dq_init done
2022-12-03 20:11:42,446 - stpipe.Detector1Pipeline.saturation - INFO - Step saturation running with args (<RampModel(1, 10, 2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:11:42,448 - stpipe.Detector1Pipeline.saturation - INFO - Step saturation parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp5lkmcroh', 'n_pix_grow_sat': 1}
2022-12-03 20:11:42,475 - stpipe.Detector1Pipeline.saturation - INFO - Using SATURATION reference file /grp/crds/cache/references/jwst/jwst_nirspec_saturation_0020.fits
2022-12-03 20:11:45,471 - stpipe.Detector1Pipeline.saturation - INFO - Detected 37655 saturated pixels
2022-12-03 20:11:45,539 - stpipe.Detector1Pipeline.saturation - INFO - Detected 0 A/D floor pixels
2022-12-03 20:11:45,559 - stpipe.Detector1Pipeline.saturation - INFO - Step saturation done
2022-12-03 20:11:45,802 - stpipe.Detector1Pipeline.ipc - INFO - Step ipc running with args (<RampModel(1, 10, 2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:11:45,804 - stpipe.Detector1Pipeline.ipc - INFO - Step ipc parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp5lkmcroh'}
2022-12-03 20:11:45,805 - stpipe.Detector1Pipeline.ipc - INFO - Step skipped.
2022-12-03 20:11:45,808 - stpipe.Detector1Pipeline.ipc - INFO - Step ipc done
2022-12-03 20:11:46,043 - stpipe.Detector1Pipeline.superbias - INFO - Step superbias running with args (<RampModel(1, 10, 2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:11:46,045 - stpipe.Detector1Pipeline.superbias - INFO - Step superbias parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp5lkmcroh'}
2022-12-03 20:11:46,075 - stpipe.Detector1Pipeline.superbias - INFO - Using SUPERBIAS reference file /grp/crds/cache/references/jwst/jwst_nirspec_superbias_0087.fits
2022-12-03 20:11:52,672 - stpipe.Detector1Pipeline.superbias - INFO - Step superbias done
2022-12-03 20:11:52,914 - stpipe.Detector1Pipeline.refpix - INFO - Step refpix running with args (<RampModel(1, 10, 2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:11:52,916 - stpipe.Detector1Pipeline.refpix - INFO - Step refpix parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp5lkmcroh', 'odd_even_columns': True, 'use_side_ref_pixels': True, 'side_smoothing_length': 11, 'side_gain': 1.0, 'odd_even_rows': True}
2022-12-03 20:11:53,113 - stpipe.Detector1Pipeline.refpix - INFO - NIR full frame data
2022-12-03 20:11:53,114 - stpipe.Detector1Pipeline.refpix - INFO - The following parameters are valid for this mode:
2022-12-03 20:11:53,114 - stpipe.Detector1Pipeline.refpix - INFO - use_side_ref_pixels = True
2022-12-03 20:11:53,114 - stpipe.Detector1Pipeline.refpix - INFO - odd_even_columns = True
2022-12-03 20:11:53,115 - stpipe.Detector1Pipeline.refpix - INFO - side_smoothing_length = 11
2022-12-03 20:11:53,115 - stpipe.Detector1Pipeline.refpix - INFO - side_gain = 1.0
2022-12-03 20:11:53,115 - stpipe.Detector1Pipeline.refpix - INFO - The following parameter is not applicable and is ignored:
2022-12-03 20:11:53,116 - stpipe.Detector1Pipeline.refpix - INFO - odd_even_rows = False
2022-12-03 20:11:58,907 - stpipe.Detector1Pipeline.refpix - INFO - Step refpix done
2022-12-03 20:11:59,156 - stpipe.Detector1Pipeline.linearity - INFO - Step linearity running with args (<RampModel(1, 10, 2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:11:59,158 - stpipe.Detector1Pipeline.linearity - INFO - Step linearity parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp5lkmcroh'}
2022-12-03 20:11:59,187 - stpipe.Detector1Pipeline.linearity - INFO - Using Linearity reference file /grp/crds/cache/references/jwst/jwst_nirspec_linearity_0018.fits
2022-12-03 20:12:00,571 - stpipe.Detector1Pipeline.linearity - INFO - Step linearity done
2022-12-03 20:12:00,723 - stpipe.Detector1Pipeline.dark_current - INFO - Step dark_current running with args (<RampModel(1, 10, 2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:12:00,725 - stpipe.Detector1Pipeline.dark_current - INFO - Step dark_current parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp5lkmcroh', 'dark_output': None}
2022-12-03 20:12:00,750 - stpipe.Detector1Pipeline.dark_current - INFO - Using DARK reference file /grp/crds/cache/references/jwst/jwst_nirspec_dark_0140.fits
2022-12-03 20:12:12,935 - stpipe.Detector1Pipeline.dark_current - INFO - Science data nints=1, ngroups=10, nframes=1, groupgap=0
2022-12-03 20:12:12,936 - stpipe.Detector1Pipeline.dark_current - INFO - Dark data nints=1, ngroups=10, nframes=1, groupgap=0
2022-12-03 20:12:13,445 - stpipe.Detector1Pipeline.dark_current - INFO - Step dark_current done
2022-12-03 20:12:13,578 - stpipe.Detector1Pipeline.jump - INFO - Step jump running with args (<RampModel(1, 10, 2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:12:13,580 - stpipe.Detector1Pipeline.jump - INFO - Step jump parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp5lkmcroh', 'rejection_threshold': 4.0, 'three_group_rejection_threshold': 6.0, 'four_group_rejection_threshold': 5.0, 'maximum_cores': 'none', 'flag_4_neighbors': True, 'max_jump_to_flag_neighbors': 1000.0, 'min_jump_to_flag_neighbors': 10.0, 'after_jump_flag_dn1': 0.0, 'after_jump_flag_time1': 0.0, 'after_jump_flag_dn2': 0.0, 'after_jump_flag_time2': 0.0, 'min_sat_area': 1.0, 'min_jump_area': 5.0, 'expand_factor': 2.0, 'use_ellipses': False, 'sat_required_snowball': True, 'expand_large_events': False}
2022-12-03 20:12:13,591 - stpipe.Detector1Pipeline.jump - INFO - CR rejection threshold = 4 sigma
2022-12-03 20:12:13,605 - stpipe.Detector1Pipeline.jump - INFO - Using GAIN reference file: /grp/crds/cache/references/jwst/jwst_nirspec_gain_0019.fits
2022-12-03 20:12:13,694 - stpipe.Detector1Pipeline.jump - INFO - Using READNOISE reference file: /grp/crds/cache/references/jwst/jwst_nirspec_readnoise_0019.fits
2022-12-03 20:12:17,968 - stpipe.Detector1Pipeline.jump - INFO - Executing two-point difference method
2022-12-03 20:12:18,074 - stpipe.Detector1Pipeline.jump - INFO - Working on integration 1:
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/jump/twopoint_difference.py:154: RuntimeWarning: All-NaN slice encountered
  max_ratio = np.nanmax(ratio, axis=0)
2022-12-03 20:12:21,933 - stpipe.Detector1Pipeline.jump - INFO - From highest outlier, two-point found 29709 pixels with at least one CR from five or more groups.
2022-12-03 20:12:26,466 - stpipe.Detector1Pipeline.jump - INFO - Total elapsed time = 8.49702 sec
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/jump/jump.py:302: RuntimeWarning: invalid value encountered in divide
  data /= gain_2d
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/jump/jump.py:303: RuntimeWarning: invalid value encountered in divide
  err /= gain_2d
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/jump/jump.py:304: RuntimeWarning: invalid value encountered in divide
  readnoise_2d /= gain_2d
2022-12-03 20:12:26,587 - stpipe.Detector1Pipeline.jump - INFO - The execution time in seconds: 12.996308
2022-12-03 20:12:26,593 - stpipe.Detector1Pipeline.jump - INFO - Step jump done
2022-12-03 20:12:26,751 - stpipe.Detector1Pipeline.ramp_fit - INFO - Step ramp_fit running with args (<RampModel(1, 10, 2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:12:26,753 - stpipe.Detector1Pipeline.ramp_fit - INFO - Step ramp_fit parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp5lkmcroh', 'int_name': '', 'save_opt': False, 'opt_name': '', 'suppress_one_group': True, 'maximum_cores': 'none'}
2022-12-03 20:12:26,792 - stpipe.Detector1Pipeline.ramp_fit - INFO - Using READNOISE reference file: /grp/crds/cache/references/jwst/jwst_nirspec_readnoise_0019.fits
2022-12-03 20:12:26,792 - stpipe.Detector1Pipeline.ramp_fit - INFO - Using GAIN reference file: /grp/crds/cache/references/jwst/jwst_nirspec_gain_0019.fits
2022-12-03 20:12:26,907 - stpipe.Detector1Pipeline.ramp_fit - INFO - Using algorithm = ols
2022-12-03 20:12:26,908 - stpipe.Detector1Pipeline.ramp_fit - INFO - Using weighting = optimal
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/stcal/ramp_fitting/ols_fit.py:1089: RuntimeWarning: invalid value encountered in multiply
  var_p4[num_int, :, :, :] *= (segs_4[num_int, :, :, :] > 0)
2022-12-03 20:13:14,873 - stpipe.Detector1Pipeline.ramp_fit - INFO - Number of groups per integration: 10
2022-12-03 20:13:14,874 - stpipe.Detector1Pipeline.ramp_fit - INFO - Number of integrations: 1
2022-12-03 20:13:15,131 - stpipe.Detector1Pipeline.ramp_fit - INFO - Step ramp_fit done
2022-12-03 20:13:15,295 - stpipe.Detector1Pipeline.gain_scale - INFO - Step gain_scale running with args (<IFUImageModel(2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:13:15,297 - stpipe.Detector1Pipeline.gain_scale - INFO - Step gain_scale parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'gain_scale', 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp5lkmcroh'}
2022-12-03 20:13:18,339 - stpipe.Detector1Pipeline.gain_scale - INFO - GAINFACT not found in gain reference file
2022-12-03 20:13:18,340 - stpipe.Detector1Pipeline.gain_scale - INFO - Step will be skipped
2022-12-03 20:13:18,345 - stpipe.Detector1Pipeline.gain_scale - INFO - Step gain_scale done
2022-12-03 20:13:18,477 - stpipe.Detector1Pipeline.gain_scale - INFO - Step gain_scale running with args (<CubeModel(1, 2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:13:18,479 - stpipe.Detector1Pipeline.gain_scale - INFO - Step gain_scale parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': 'gain_scaleints', 'search_output_file': True, 'input_dir': '/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp5lkmcroh'}
2022-12-03 20:13:18,562 - stpipe.Detector1Pipeline.gain_scale - INFO - GAINFACT not found in gain reference file
2022-12-03 20:13:18,564 - stpipe.Detector1Pipeline.gain_scale - INFO - Step will be skipped
2022-12-03 20:13:18,569 - stpipe.Detector1Pipeline.gain_scale - INFO - Step gain_scale done
2022-12-03 20:13:18,570 - stpipe.Detector1Pipeline - INFO - ... ending calwebb_detector1
2022-12-03 20:13:18,571 - stpipe.Detector1Pipeline - INFO - Results used CRDS context: jwst_1019.pmap
2022-12-03 20:13:18,571 - stpipe.Detector1Pipeline - INFO - Step Detector1Pipeline done
2022-12-03 20:13:18,611 - CRDS - ERROR -  Error determining best reference for 'pars-residualfringestep'  =   Unknown reference type 'pars-residualfringestep'
2022-12-03 20:13:18,629 - stpipe.Spec2Pipeline - INFO - Spec2Pipeline instance created.
2022-12-03 20:13:18,631 - stpipe.Spec2Pipeline.bkg_subtract - INFO - BackgroundStep instance created.
2022-12-03 20:13:18,632 - stpipe.Spec2Pipeline.assign_wcs - INFO - AssignWcsStep instance created.
2022-12-03 20:13:18,633 - stpipe.Spec2Pipeline.imprint_subtract - INFO - ImprintStep instance created.
2022-12-03 20:13:18,634 - stpipe.Spec2Pipeline.msa_flagging - INFO - MSAFlagOpenStep instance created.
2022-12-03 20:13:18,636 - stpipe.Spec2Pipeline.extract_2d - INFO - Extract2dStep instance created.
2022-12-03 20:13:18,639 - stpipe.Spec2Pipeline.master_background_mos - INFO - MasterBackgroundMosStep instance created.
2022-12-03 20:13:18,640 - stpipe.Spec2Pipeline.master_background_mos.flat_field - INFO - FlatFieldStep instance created.
2022-12-03 20:13:18,641 - stpipe.Spec2Pipeline.master_background_mos.pathloss - INFO - PathLossStep instance created.
2022-12-03 20:13:18,642 - stpipe.Spec2Pipeline.master_background_mos.barshadow - INFO - BarShadowStep instance created.
2022-12-03 20:13:18,643 - stpipe.Spec2Pipeline.master_background_mos.photom - INFO - PhotomStep instance created.
2022-12-03 20:13:18,644 - stpipe.Spec2Pipeline.wavecorr - INFO - WavecorrStep instance created.
2022-12-03 20:13:18,645 - stpipe.Spec2Pipeline.flat_field - INFO - FlatFieldStep instance created.
2022-12-03 20:13:18,646 - stpipe.Spec2Pipeline.srctype - INFO - SourceTypeStep instance created.
2022-12-03 20:13:18,647 - stpipe.Spec2Pipeline.straylight - INFO - StraylightStep instance created.
2022-12-03 20:13:18,649 - stpipe.Spec2Pipeline.fringe - INFO - FringeStep instance created.
2022-12-03 20:13:18,650 - stpipe.Spec2Pipeline.residual_fringe - INFO - ResidualFringeStep instance created.
2022-12-03 20:13:18,651 - stpipe.Spec2Pipeline.pathloss - INFO - PathLossStep instance created.
2022-12-03 20:13:18,652 - stpipe.Spec2Pipeline.barshadow - INFO - BarShadowStep instance created.
2022-12-03 20:13:18,653 - stpipe.Spec2Pipeline.wfss_contam - INFO - WfssContamStep instance created.
2022-12-03 20:13:18,654 - stpipe.Spec2Pipeline.photom - INFO - PhotomStep instance created.
2022-12-03 20:13:18,656 - stpipe.Spec2Pipeline.resample_spec - INFO - ResampleSpecStep instance created.
2022-12-03 20:13:18,657 - stpipe.Spec2Pipeline.cube_build - INFO - CubeBuildStep instance created.
2022-12-03 20:13:18,659 - stpipe.Spec2Pipeline.extract_1d - INFO - Extract1dStep instance created.

Running the spec2 pipeline...

2022-12-03 20:13:18,818 - stpipe.Spec2Pipeline - INFO - Step Spec2Pipeline running with args (<IFUImageModel(2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:13:18,834 - stpipe.Spec2Pipeline - INFO - Step Spec2Pipeline parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_bsub': False, 'fail_on_exception': True, 'save_wfss_esec': False, 'steps': {'bkg_subtract': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_combined_background': False, 'sigma': 3.0, 'maxiters': None, 'wfss_mmag_extract': None}, 'assign_wcs': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'sip_approx': True, 'sip_max_pix_error': 0.1, 'sip_degree': None, 'sip_max_inv_pix_error': 0.1, 'sip_inv_degree': None, 'sip_npoints': 12, 'slit_y_low': -0.55, 'slit_y_high': 0.55}, 'imprint_subtract': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'msa_flagging': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'extract_2d': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'slit_name': None, 'extract_orders': None, 'grism_objects': None, 'tsgrism_extract_height': None, 'wfss_extract_half_height': 5, 'wfss_mmag_extract': None, 'wfss_nbright': 1000}, 'master_background_mos': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'force_subtract': False, 'save_background': False, 'user_background': None, 'inverse': False, 'steps': {'flat_field': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_interpolated_flat': False, 'user_supplied_flat': None, 'inverse': False}, 'pathloss': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'inverse': False, 'source_type': None}, 'barshadow': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'inverse': False, 'source_type': None}, 'photom': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'inverse': False, 'source_type': None}}}, 'wavecorr': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'flat_field': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_interpolated_flat': True, 'user_supplied_flat': None, 'inverse': False}, 'srctype': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'source_type': None}, 'straylight': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'fringe': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}, 'residual_fringe': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'residual_fringe', 'search_output_file': False, 'input_dir': '', 'save_intermediate_results': False, 'ignore_region_min': None, 'ignore_region_max': None}, 'pathloss': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'inverse': False, 'source_type': None}, 'barshadow': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'inverse': False, 'source_type': None}, 'wfss_contam': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_simulated_image': False, 'save_contam_images': False, 'maximum_cores': 'none'}, 'photom': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'inverse': False, 'source_type': None}, 'resample_spec': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'pixfrac': 1.0, 'kernel': 'square', 'fillval': 'INDEF', 'weight_type': 'ivm', 'output_shape': None, 'crpix': None, 'crval': None, 'rotation': None, 'pixel_scale_ratio': 1.0, 'pixel_scale': None, 'single': False, 'blendheaders': True, 'allowed_memory': None, 'in_memory': True}, 'cube_build': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'channel': 'all', 'band': 'all', 'grating': 'all', 'filter': 'all', 'output_type': 'band', 'scale1': 0.0, 'scale2': 0.0, 'scalew': 0.0, 'weighting': 'drizzle', 'coord_system': 'skyalign', 'rois': 0.0, 'roiw': 0.0, 'weight_power': 2.0, 'wavemin': None, 'wavemax': None, 'single': False, 'skip_dqflagging': False}, 'extract_1d': {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'smoothing_length': None, 'bkg_fit': None, 'bkg_order': None, 'bkg_sigma_clip': 3.0, 'log_increment': 50, 'subtract_background': None, 'use_source_posn': None, 'center_xy': None, 'apply_apcorr': True, 'soss_threshold': 0.01, 'soss_n_os': 2, 'soss_transform': None, 'soss_tikfac': None, 'soss_width': 40.0, 'soss_bad_pix': 'model', 'soss_modelname': None}}}
2022-12-03 20:13:18,845 - stpipe.Spec2Pipeline - INFO - Prefetching reference files for dataset: 'ifu_prism_nrs1_uncal.fits' reftypes = ['camera', 'collimator', 'dflat', 'disperser', 'distortion', 'fflat', 'filteroffset', 'flat', 'fore', 'fpa', 'fringe', 'ifufore', 'ifupost', 'ifuslicer', 'mrsxartcorr', 'msa', 'msaoper', 'ote', 'regions', 'sflat', 'specwcs', 'wavecorr', 'wavelengthrange', 'wfssbkg']
2022-12-03 20:13:18,890 - stpipe.Spec2Pipeline - INFO - Prefetch for CAMERA reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_camera_0004.asdf'.
2022-12-03 20:13:18,899 - stpipe.Spec2Pipeline - INFO - Prefetch for COLLIMATOR reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_collimator_0004.asdf'.
2022-12-03 20:13:18,908 - stpipe.Spec2Pipeline - INFO - Prefetch for DFLAT reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_dflat_0001.fits'.
2022-12-03 20:13:18,917 - stpipe.Spec2Pipeline - INFO - Prefetch for DISPERSER reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_disperser_0034.asdf'.
2022-12-03 20:13:18,920 - stpipe.Spec2Pipeline - INFO - Prefetch for DISTORTION reference file is 'N/A'.
2022-12-03 20:13:18,920 - stpipe.Spec2Pipeline - INFO - Prefetch for FFLAT reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_fflat_0007.fits'.
2022-12-03 20:13:18,927 - stpipe.Spec2Pipeline - INFO - Prefetch for FILTEROFFSET reference file is 'N/A'.
2022-12-03 20:13:18,927 - stpipe.Spec2Pipeline - INFO - Prefetch for FLAT reference file is 'N/A'.
2022-12-03 20:13:18,928 - stpipe.Spec2Pipeline - INFO - Prefetch for FORE reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_fore_0028.asdf'.
2022-12-03 20:13:18,933 - stpipe.Spec2Pipeline - INFO - Prefetch for FPA reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_fpa_0005.asdf'.
2022-12-03 20:13:18,936 - stpipe.Spec2Pipeline - INFO - Prefetch for FRINGE reference file is 'N/A'.
2022-12-03 20:13:18,936 - stpipe.Spec2Pipeline - INFO - Prefetch for IFUFORE reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_ifufore_0003.asdf'.
2022-12-03 20:13:18,952 - stpipe.Spec2Pipeline - INFO - Prefetch for IFUPOST reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_ifupost_0004.asdf'.
2022-12-03 20:13:18,962 - stpipe.Spec2Pipeline - INFO - Prefetch for IFUSLICER reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_ifuslicer_0003.asdf'.
2022-12-03 20:13:18,966 - stpipe.Spec2Pipeline - INFO - Prefetch for MRSXARTCORR reference file is 'N/A'.
2022-12-03 20:13:18,967 - stpipe.Spec2Pipeline - INFO - Prefetch for MSA reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_msa_0005.asdf'.
2022-12-03 20:13:18,970 - stpipe.Spec2Pipeline - INFO - Prefetch for MSAOPER reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_msaoper_0001.json'.
2022-12-03 20:13:18,982 - stpipe.Spec2Pipeline - INFO - Prefetch for OTE reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_ote_0005.asdf'.
2022-12-03 20:13:18,994 - stpipe.Spec2Pipeline - INFO - Prefetch for REGIONS reference file is 'N/A'.
2022-12-03 20:13:18,994 - stpipe.Spec2Pipeline - INFO - Prefetch for SFLAT reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_sflat_0004.fits'.
2022-12-03 20:13:18,998 - stpipe.Spec2Pipeline - INFO - Prefetch for SPECWCS reference file is 'N/A'.
2022-12-03 20:13:18,999 - stpipe.Spec2Pipeline - INFO - Prefetch for WAVECORR reference file is 'N/A'.
2022-12-03 20:13:18,999 - stpipe.Spec2Pipeline - INFO - Prefetch for WAVELENGTHRANGE reference file is '/grp/crds/cache/references/jwst/jwst_nirspec_wavelengthrange_0004.asdf'.
2022-12-03 20:13:19,008 - stpipe.Spec2Pipeline - INFO - Prefetch for WFSSBKG reference file is 'N/A'.
2022-12-03 20:13:19,009 - stpipe.Spec2Pipeline - INFO - Starting calwebb_spec2 ...
2022-12-03 20:13:19,070 - stpipe.Spec2Pipeline - INFO - Processing product ifu_prism_nrs1_uncal
2022-12-03 20:13:19,071 - stpipe.Spec2Pipeline - INFO - Working on input <IFUImageModel(2048, 2048) from ifu_prism_nrs1_uncal.fits> ...
2022-12-03 20:13:19,212 - stpipe.Spec2Pipeline.assign_wcs - INFO - Step assign_wcs running with args (<IFUImageModel(2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:13:19,215 - stpipe.Spec2Pipeline.assign_wcs - INFO - Step assign_wcs parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'sip_approx': True, 'sip_max_pix_error': 0.1, 'sip_degree': None, 'sip_max_inv_pix_error': 0.1, 'sip_inv_degree': None, 'sip_npoints': 12, 'slit_y_low': -0.55, 'slit_y_high': 0.55}
2022-12-03 20:13:19,520 - stpipe.Spec2Pipeline.assign_wcs - INFO - gwa_ytilt is 0.03312480077147484 deg
2022-12-03 20:13:19,521 - stpipe.Spec2Pipeline.assign_wcs - INFO - gwa_xtilt is 0.3411945700645447 deg
2022-12-03 20:13:19,522 - stpipe.Spec2Pipeline.assign_wcs - INFO - theta_y correction: -0.005294283663966503 deg
2022-12-03 20:13:19,523 - stpipe.Spec2Pipeline.assign_wcs - INFO - theta_x correction: 0.0 deg
2022-12-03 20:13:23,279 - stpipe.Spec2Pipeline.assign_wcs - INFO - Created a NIRSPEC nrs_ifu pipeline with references {'distortion': None, 'filteroffset': None, 'specwcs': None, 'regions': None, 'wavelengthrange': '/grp/crds/cache/references/jwst/jwst_nirspec_wavelengthrange_0004.asdf', 'camera': '/grp/crds/cache/references/jwst/jwst_nirspec_camera_0004.asdf', 'collimator': '/grp/crds/cache/references/jwst/jwst_nirspec_collimator_0004.asdf', 'disperser': '/grp/crds/cache/references/jwst/jwst_nirspec_disperser_0034.asdf', 'fore': '/grp/crds/cache/references/jwst/jwst_nirspec_fore_0028.asdf', 'fpa': '/grp/crds/cache/references/jwst/jwst_nirspec_fpa_0005.asdf', 'msa': '/grp/crds/cache/references/jwst/jwst_nirspec_msa_0005.asdf', 'ote': '/grp/crds/cache/references/jwst/jwst_nirspec_ote_0005.asdf', 'ifupost': '/grp/crds/cache/references/jwst/jwst_nirspec_ifupost_0004.asdf', 'ifufore': '/grp/crds/cache/references/jwst/jwst_nirspec_ifufore_0003.asdf', 'ifuslicer': '/grp/crds/cache/references/jwst/jwst_nirspec_ifuslicer_0003.asdf'}
2022-12-03 20:13:28,972 - stpipe.Spec2Pipeline.assign_wcs - INFO - Update S_REGION to POLYGON ICRS  156.176998032 -45.687618307 156.178780407 -45.687618307 156.178780407 -45.686330486 156.176998032 -45.686330486
2022-12-03 20:13:28,975 - stpipe.Spec2Pipeline.assign_wcs - INFO - COMPLETED assign_wcs
2022-12-03 20:13:28,989 - stpipe.Spec2Pipeline.assign_wcs - INFO - Step assign_wcs done
2022-12-03 20:13:29,478 - stpipe.Spec2Pipeline.bkg_subtract - INFO - Step bkg_subtract running with args (<IFUImageModel(2048, 2048) from ifu_prism_nrs1_uncal.fits>, []).
2022-12-03 20:13:29,480 - stpipe.Spec2Pipeline.bkg_subtract - INFO - Step bkg_subtract parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_combined_background': False, 'sigma': 3.0, 'maxiters': None, 'wfss_mmag_extract': None}
2022-12-03 20:13:29,481 - stpipe.Spec2Pipeline.bkg_subtract - INFO - Step skipped.
2022-12-03 20:13:29,484 - stpipe.Spec2Pipeline.bkg_subtract - INFO - Step bkg_subtract done
2022-12-03 20:13:29,839 - stpipe.Spec2Pipeline.imprint_subtract - INFO - Step imprint_subtract running with args (<IFUImageModel(2048, 2048) from ifu_prism_nrs1_uncal.fits>, []).
2022-12-03 20:13:29,841 - stpipe.Spec2Pipeline.imprint_subtract - INFO - Step imprint_subtract parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': ''}
2022-12-03 20:13:29,841 - stpipe.Spec2Pipeline.imprint_subtract - INFO - Step skipped.
2022-12-03 20:13:29,844 - stpipe.Spec2Pipeline.imprint_subtract - INFO - Step imprint_subtract done
2022-12-03 20:13:30,219 - stpipe.Spec2Pipeline.msa_flagging - INFO - Step msa_flagging running with args (<IFUImageModel(2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:13:30,221 - stpipe.Spec2Pipeline.msa_flagging - INFO - Step msa_flagging parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': ''}
2022-12-03 20:13:30,253 - stpipe.Spec2Pipeline.msa_flagging - INFO - Using reference file /grp/crds/cache/references/jwst/jwst_nirspec_msaoper_0001.json
2022-12-03 20:13:30,255 - stpipe.JwstStep - INFO - JwstStep instance created.
2022-12-03 20:13:30,568 - stpipe.Spec2Pipeline.msa_flagging - INFO - gwa_ytilt is 0.03312480077147484 deg
2022-12-03 20:13:30,569 - stpipe.Spec2Pipeline.msa_flagging - INFO - gwa_xtilt is 0.3411945700645447 deg
2022-12-03 20:13:30,570 - stpipe.Spec2Pipeline.msa_flagging - INFO - theta_y correction: -0.005294283663966503 deg
2022-12-03 20:13:30,572 - stpipe.Spec2Pipeline.msa_flagging - INFO - theta_x correction: 0.0 deg
2022-12-03 20:13:30,589 - stpipe.Spec2Pipeline.msa_flagging - INFO - SPORDER= 0, wrange=[6e-07, 5.3e-06]
2022-12-03 20:13:30,820 - stpipe.Spec2Pipeline.msa_flagging - INFO - There are 5 open slits in quadrant 1
2022-12-03 20:13:30,897 - stpipe.Spec2Pipeline.msa_flagging - INFO - There are 3 open slits in quadrant 2
2022-12-03 20:13:30,931 - stpipe.Spec2Pipeline.msa_flagging - INFO - There are 9 open slits in quadrant 3
2022-12-03 20:13:31,023 - stpipe.Spec2Pipeline.msa_flagging - INFO - There are 3 open slits in quadrant 4
2022-12-03 20:13:31,055 - stpipe.Spec2Pipeline.msa_flagging - INFO - There are 0 open slits in quadrant 5
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/astropy/modeling/bounding_box.py:83: RuntimeWarning: Invalid interval: upper bound 2047.5 is strictly less than lower bound 2585.2572652943927.
  warnings.warn(f"Invalid interval: upper bound {upper} "
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/astropy/modeling/bounding_box.py:83: RuntimeWarning: Invalid interval: upper bound 2047.5 is strictly less than lower bound 2329.91715711254.
  warnings.warn(f"Invalid interval: upper bound {upper} "
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/astropy/modeling/bounding_box.py:83: RuntimeWarning: Invalid interval: upper bound 2047.5 is strictly less than lower bound 2731.341633999907.
  warnings.warn(f"Invalid interval: upper bound {upper} "
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/astropy/modeling/bounding_box.py:83: RuntimeWarning: Invalid interval: upper bound 2047.5 is strictly less than lower bound 2095.6529334650595.
  warnings.warn(f"Invalid interval: upper bound {upper} "
/internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/miniconda3/envs/jwst_validation_notebooks/lib/python3.9/site-packages/astropy/modeling/bounding_box.py:83: RuntimeWarning: Invalid interval: upper bound 2047.5 is strictly less than lower bound 2085.8983001463794.
  warnings.warn(f"Invalid interval: upper bound {upper} "
2022-12-03 20:13:46,936 - stpipe.Spec2Pipeline.msa_flagging - INFO - Step msa_flagging done
2022-12-03 20:13:47,482 - stpipe.Spec2Pipeline.srctype - INFO - Step srctype running with args (<IFUImageModel(2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:13:47,484 - stpipe.Spec2Pipeline.srctype - INFO - Step srctype parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'source_type': None}
2022-12-03 20:13:47,493 - stpipe.Spec2Pipeline.srctype - INFO - Input EXP_TYPE is NRS_IFU
2022-12-03 20:13:47,494 - stpipe.Spec2Pipeline.srctype - INFO - Input SRCTYAPT = UNKNOWN
2022-12-03 20:13:47,495 - stpipe.Spec2Pipeline.srctype - INFO - Input source type is unknown; setting default SRCTYPE = EXTENDED
2022-12-03 20:13:47,498 - stpipe.Spec2Pipeline.srctype - INFO - Step srctype done
2022-12-03 20:13:47,708 - stpipe.Spec2Pipeline.flat_field - INFO - Step flat_field running with args (<IFUImageModel(2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:13:47,710 - stpipe.Spec2Pipeline.flat_field - INFO - Step flat_field parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': False, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'save_interpolated_flat': True, 'user_supplied_flat': None, 'inverse': False}
2022-12-03 20:16:56,245 - stpipe.Spec2Pipeline.flat_field - INFO - Saved model in ifu_prism_nrs1_interpolatedflat.fits
2022-12-03 20:16:56,247 - stpipe.Spec2Pipeline.flat_field - INFO - Interpolated flat written to "ifu_prism_nrs1_interpolatedflat.fits".
2022-12-03 20:16:56,254 - stpipe.Spec2Pipeline.flat_field - INFO - Step flat_field done
2022-12-03 20:16:57,426 - stpipe.Spec2Pipeline.straylight - INFO - Step straylight running with args (<IFUImageModel(2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:16:57,428 - stpipe.Spec2Pipeline.straylight - INFO - Step straylight parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': ''}
2022-12-03 20:16:57,429 - stpipe.Spec2Pipeline.straylight - INFO - Step skipped.
2022-12-03 20:16:57,432 - stpipe.Spec2Pipeline.straylight - INFO - Step straylight done
2022-12-03 20:16:57,768 - stpipe.Spec2Pipeline.fringe - INFO - Step fringe running with args (<IFUImageModel(2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:16:57,770 - stpipe.Spec2Pipeline.fringe - INFO - Step fringe parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': ''}
2022-12-03 20:16:57,770 - stpipe.Spec2Pipeline.fringe - INFO - Step skipped.
2022-12-03 20:16:57,773 - stpipe.Spec2Pipeline.fringe - INFO - Step fringe done
2022-12-03 20:16:58,094 - stpipe.Spec2Pipeline.pathloss - INFO - Step pathloss running with args (<IFUImageModel(2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:16:58,095 - stpipe.Spec2Pipeline.pathloss - INFO - Step pathloss parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'inverse': False, 'source_type': None}
2022-12-03 20:16:58,096 - stpipe.Spec2Pipeline.pathloss - INFO - Step skipped.
2022-12-03 20:16:58,099 - stpipe.Spec2Pipeline.pathloss - INFO - Step pathloss done
2022-12-03 20:16:58,414 - stpipe.Spec2Pipeline.barshadow - INFO - Step barshadow running with args (<IFUImageModel(2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:16:58,416 - stpipe.Spec2Pipeline.barshadow - INFO - Step barshadow parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'inverse': False, 'source_type': None}
2022-12-03 20:16:58,417 - stpipe.Spec2Pipeline.barshadow - INFO - Step skipped.
2022-12-03 20:16:58,420 - stpipe.Spec2Pipeline.barshadow - INFO - Step barshadow done
2022-12-03 20:16:58,756 - stpipe.Spec2Pipeline.photom - INFO - Step photom running with args (<IFUImageModel(2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:16:58,758 - stpipe.Spec2Pipeline.photom - INFO - Step photom parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': True, 'input_dir': '', 'inverse': False, 'source_type': None}
2022-12-03 20:16:58,759 - stpipe.Spec2Pipeline.photom - INFO - Step skipped.
2022-12-03 20:16:58,762 - stpipe.Spec2Pipeline.photom - INFO - Step photom done
2022-12-03 20:16:59,126 - stpipe.Spec2Pipeline.residual_fringe - INFO - Step residual_fringe running with args (<IFUImageModel(2048, 2048) from ifu_prism_nrs1_uncal.fits>,).
2022-12-03 20:16:59,129 - stpipe.Spec2Pipeline.residual_fringe - INFO - Step residual_fringe parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'residual_fringe', 'search_output_file': False, 'input_dir': '', 'save_intermediate_results': False, 'ignore_region_min': None, 'ignore_region_max': None}
2022-12-03 20:16:59,129 - stpipe.Spec2Pipeline.residual_fringe - INFO - Step skipped.
2022-12-03 20:16:59,132 - stpipe.Spec2Pipeline.residual_fringe - INFO - Step residual_fringe done
2022-12-03 20:16:59,540 - stpipe.Spec2Pipeline.cube_build - INFO - Step cube_build running with args (<IFUImageModel(2048, 2048) from ifu_prism_nrs1_cal.fits>,).
2022-12-03 20:16:59,542 - stpipe.Spec2Pipeline.cube_build - INFO - Step cube_build parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': True, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': None, 'search_output_file': False, 'input_dir': '', 'channel': 'all', 'band': 'all', 'grating': 'all', 'filter': 'all', 'output_type': 'multi', 'scale1': 0.0, 'scale2': 0.0, 'scalew': 0.0, 'weighting': 'drizzle', 'coord_system': 'skyalign', 'rois': 0.0, 'roiw': 0.0, 'weight_power': 2.0, 'wavemin': None, 'wavemax': None, 'single': False, 'skip_dqflagging': True}
2022-12-03 20:16:59,543 - stpipe.Spec2Pipeline.cube_build - INFO - Step skipped.
2022-12-03 20:16:59,546 - stpipe.Spec2Pipeline.cube_build - INFO - Step cube_build done
2022-12-03 20:16:59,798 - stpipe.Spec2Pipeline.extract_1d - INFO - Step extract_1d running with args (<IFUImageModel(2048, 2048) from ifu_prism_nrs1_cal.fits>,).
2022-12-03 20:16:59,800 - stpipe.Spec2Pipeline.extract_1d - INFO - Step extract_1d parameters are: {'pre_hooks': [], 'post_hooks': [], 'output_file': None, 'output_dir': None, 'output_ext': '.fits', 'output_use_model': False, 'output_use_index': True, 'save_results': False, 'skip': True, 'suffix': 'x1d', 'search_output_file': True, 'input_dir': '', 'smoothing_length': None, 'bkg_fit': None, 'bkg_order': None, 'bkg_sigma_clip': 3.0, 'log_increment': 50, 'subtract_background': None, 'use_source_posn': None, 'center_xy': None, 'apply_apcorr': True, 'soss_threshold': 0.01, 'soss_n_os': 2, 'soss_transform': None, 'soss_tikfac': None, 'soss_width': 40.0, 'soss_bad_pix': 'model', 'soss_modelname': None}
2022-12-03 20:16:59,801 - stpipe.Spec2Pipeline.extract_1d - INFO - Step skipped.
2022-12-03 20:16:59,803 - stpipe.Spec2Pipeline.extract_1d - INFO - Step extract_1d done
2022-12-03 20:16:59,804 - stpipe.Spec2Pipeline - INFO - Finished processing product ifu_prism_nrs1_uncal
2022-12-03 20:16:59,804 - stpipe.Spec2Pipeline - INFO - Ending calwebb_spec2
2022-12-03 20:16:59,805 - stpipe.Spec2Pipeline - INFO - Results used CRDS context: jwst_1019.pmap
2022-12-03 20:16:59,805 - stpipe.Spec2Pipeline - INFO - Step Spec2Pipeline done

Running flat field test for IFU...
flat_field_file  -->     Grating:PRISM   Filter:CLEAR   LAMP:NO_LAMP
Getting and reading the D-, S-, and F-flats for this specific IFU configuration... 
Using D-flat: /internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp5lkmcroh/nirspec_dflat_nrs1_f_01.03.fits
Using S-flat: /internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp5lkmcroh/nirspec_IFU_sflat_PRISM_OPAQUE_FLAT5_nrs1_f_01.01.fits
Using F-flat: /internal/data1/jenkins/workspace/Notebooks/jwst_validation_notebooks_spacetelescope/tmp/tmp5lkmcroh/nirspec_IFU_fflat_CLEAR_01.01.fits

 Now looping through the slices, this may take some time... 

Working with slice: 00
 Subwindow origin:   px0=562   py0=781
 Looping through the wavelength, this may take a little time ... 
Flat value differences for slice number: 00

   Absolute Flat Difference  :   mean = -5.585e-06    median = -1.598e-06    stdev = 4.728e-04
    Maximum   AbsoluteFlat Difference = 1.138e-02
    Minimum   AbsoluteFlat Difference = -8.209e-03
    Percentage of pixels where median of absolute differences is greater than: 
                            ->  1xtheshold = 0%
                            ->  3xtheshold = 0%
                            ->  5xtheshold = 0%
Making the plot for this slice...

 *** Result of the test: PASSED


Working with slice: 01
 Subwindow origin:   px0=506   py0=1198
 Looping through the wavelength, this may take a little time ... 
Flat value differences for slice number: 01

   Absolute Flat Difference  :   mean = 2.032e-06    median = 1.286e-06    stdev = 4.612e-04
    Maximum   AbsoluteFlat Difference = 6.883e-03
    Minimum   AbsoluteFlat Difference = -8.178e-03
    Percentage of pixels where median of absolute differences is greater than: 
                            ->  1xtheshold = 0%
                            ->  3xtheshold = 0%
                            ->  5xtheshold = 0%
Making the plot for this slice...

 *** Result of the test: PASSED


Working with slice: 02
 Subwindow origin:   px0=568   py0=731
 Looping through the wavelength, this may take a little time ... 
Flat value differences for slice number: 02

   Absolute Flat Difference  :   mean = 7.401e-06    median = 4.806e-06    stdev = 4.664e-04
    Maximum   AbsoluteFlat Difference = 5.417e-03
    Minimum   AbsoluteFlat Difference = -8.054e-03
    Percentage of pixels where median of absolute differences is greater than: 
                            ->  1xtheshold = 0%
                            ->  3xtheshold = 0%
                            ->  5xtheshold = 0%
Making the plot for this slice...

 *** Result of the test: PASSED


Working with slice: 03
 Subwindow origin:   px0=499   py0=1247
 Looping through the wavelength, this may take a little time ... 
Flat value differences for slice number: 03

   Absolute Flat Difference  :   mean = 1.898e-06    median = 1.814e-07    stdev = 4.683e-04
    Maximum   AbsoluteFlat Difference = 6.185e-03
    Minimum   AbsoluteFlat Difference = -7.354e-03
    Percentage of pixels where median of absolute differences is greater than: 
                            ->  1xtheshold = 0%
                            ->  3xtheshold = 0%
                            ->  5xtheshold = 0%
Making the plot for this slice...

 *** Result of the test: PASSED


Working with slice: 04
 Subwindow origin:   px0=574   py0=682
 Looping through the wavelength, this may take a little time ... 
Flat value differences for slice number: 04

   Absolute Flat Difference  :   mean = -5.143e-06    median = 6.561e-07    stdev = 4.649e-04
    Maximum   AbsoluteFlat Difference = 8.423e-03
    Minimum   AbsoluteFlat Difference = -1.048e-02
    Percentage of pixels where median of absolute differences is greater than: 
                            ->  1xtheshold = 0%
                            ->  3xtheshold = 0%
                            ->  5xtheshold = 0%
Making the plot for this slice...

 *** Result of the test: PASSED


Working with slice: 05
 Subwindow origin:   px0=492   py0=1296
 Looping through the wavelength, this may take a little time ... 
Flat value differences for slice number: 05

   Absolute Flat Difference  :   mean = 8.554e-07    median = 2.257e-06    stdev = 4.876e-04
    Maximum   AbsoluteFlat Difference = 4.968e-03
    Minimum   AbsoluteFlat Difference = -9.628e-03
    Percentage of pixels where median of absolute differences is greater than: 
                            ->  1xtheshold = 0%
                            ->  3xtheshold = 0%
                            ->  5xtheshold = 0%
Making the plot for this slice...


# Quickly see if the test passed 

print('These are the final results of the tests: ')
for key, val in results_dict.items():
    if not isinstance(val, str):
        if val:
            val = 'PASSED'
        else:
            val = 'FAILED'
    print('{:<42} {:<8}'.format(key, val))

These are the final results of the tests: 
ifu_prism_nrs1_uncal.fits                  PASSED  
ifu_prism_nrs2_uncal.fits                  skipped

JWST Pipeline Validation Testing Notebook: spec2, flat_field step for ifu prism¶

Table of Contents¶

Imports¶

Test Description¶

Test Results¶

Calibration WG Requested Algorithm:¶

Defining Term¶

Run the JWST pipeline and assign_wcs validation tests¶

Testing Data Set¶

About this Notebook¶